{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "from tqdm import tqdm\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 一、特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1、合并训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_data():\n",
    "    # 标签\n",
    "    train_clicks = pd.read_csv(\"data_set_phase1//train_clicks.csv\")\n",
    "\n",
    "    # 特征数据集\n",
    "    train_plans = pd.read_csv(\"data_set_phase1//train_plans.csv\")\n",
    "    train_queries = pd.read_csv(\"data_set_phase1//train_queries.csv\")\n",
    "    test_plans = pd.read_csv(\"data_set_phase1//test_plans.csv\")\n",
    "    test_queries = pd.read_csv(\"data_set_phase1//test_queries.csv\")\n",
    "\n",
    "    # merge训练集\n",
    "    tra_data = train_queries.merge(train_plans, on='sid', how='left')\n",
    "    tra_data = tra_data.merge(train_clicks, on='sid', how='left')\n",
    "    tra_data = tra_data.drop(['click_time'], axis=1)\n",
    "    tra_data['click_mode'] = tra_data['click_mode'].fillna(0)\n",
    "\n",
    "    # merge测试集\n",
    "    tes_data = test_queries.merge(test_plans, on='sid', how='left')\n",
    "    tes_data['click_mode'] = -1\n",
    "\n",
    "    # concat训练集和测试集\n",
    "    all_data = pd.concat([tra_data, tes_data], axis=0)\n",
    "    all_data = all_data.drop(['plan_time'], axis=1)\n",
    "    all_data = all_data.reset_index(drop=True)\n",
    "    \n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2、抽取o、d的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_od_feature(all_data):\n",
    "    all_data['o1'] = all_data['o'].apply(lambda x : float(x.split(',')[0]))\n",
    "    all_data['o2'] = all_data['o'].apply(lambda x : float(x.split(',')[1]))\n",
    "    all_data['d1'] = all_data['d'].apply(lambda x : float(x.split(',')[0]))\n",
    "    all_data['d2'] = all_data['d'].apply(lambda x : float(x.split(',')[1]))\n",
    "    all_data = all_data.drop(['o', 'd'], axis=1)\n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3、抽取plans的特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 提取plans特征\n",
    "### 1、max_distance、min_distance、mean_distance、std_distance\n",
    "### 2、max_price、min_price、mean_price、std_price\n",
    "### 3、max_eta、min_eta、mean_eta、std_eta\n",
    "### 4、max_dis_mode、min_dis_mode、max_price_mode、min_price_mode、max_eta_mode、min_eta_mode\n",
    "### 5、first_mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_plan_feature(all_data):\n",
    "    n = all_data.shape[0]\n",
    "    \n",
    "    # 初始化推荐给用户的plans，类似于one-hot编码，推荐了哪一个mode，就置为1\n",
    "    mode_list_feas = np.zeros((n, 12))\n",
    "\n",
    "    # 初始化最大距离、最小距离、平均距离、距离标准差\n",
    "    max_distance, min_distance, mean_distance, std_distance = np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,))\n",
    "\n",
    "    # 初始化最大价格、最小价格、平均价格、价格标准差\n",
    "    max_price, min_price, mean_price, std_price = np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,))\n",
    "\n",
    "    # 初始化最大用时、最小用时、平均用时、用时标准差\n",
    "    max_eta, min_eta, mean_eta, std_eta = np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,))\n",
    "\n",
    "    # 初始化最大距离mode、最小距离mode、最大价格mode、最小价格mode、最大用时mode、最小用时mode、第一推荐mode\n",
    "    max_dis_mode, min_dis_mode, max_price_mode, min_price_mode, max_eta_mode, min_eta_mode, first_mode = np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,)),np.zeros((n,))\n",
    "\n",
    "    # 初始化推荐mode的顺序\n",
    "    mode_texts=[]\n",
    "    \n",
    "    # 遍历每个用户的plan\n",
    "    for i, plan in tqdm(enumerate(all_data['plans'].values)):\n",
    "        try:\n",
    "            user_plan_list = json.loads(plan)\n",
    "        except:\n",
    "            user_plan_list = []\n",
    "        if len(user_plan_list)==0:\n",
    "            mode_list_feas[i, 0] = 1\n",
    "\n",
    "            first_mode[i] = 0\n",
    "\n",
    "            max_distance[i] = -1\n",
    "            min_distance[i] = -1\n",
    "            mean_distance[i] = -1\n",
    "            std_distance[i] = -1\n",
    "\n",
    "            max_price[i] = -1\n",
    "            min_price[i] = -1\n",
    "            mean_price[i] = -1\n",
    "            std_price[i] = -1\n",
    "\n",
    "            max_eta[i] = -1\n",
    "            min_eta[i] = -1\n",
    "            mean_eta[i] = -1\n",
    "            std_eta[i] = -1\n",
    "\n",
    "            max_dis_mode[i] = -1\n",
    "            min_dis_mode[i] = -1\n",
    "            max_price_mode[i] = -1\n",
    "            min_price_mode[i] = -1\n",
    "            max_eta_mode[i] = -1\n",
    "            min_eta_mode[i] = -1\n",
    "\n",
    "            mode_texts.append('word_null')\n",
    "        else:\n",
    "            distance_list = []\n",
    "            price_list = []\n",
    "            eta_list = []\n",
    "            mode_list = []\n",
    "\n",
    "            # 抽取每个用户的每个plan\n",
    "            for tmp_dict in user_plan_list:\n",
    "                distance_list.append(int(tmp_dict['distance']))\n",
    "                if tmp_dict['price']=='':\n",
    "                    price_list.append(0)\n",
    "                else:\n",
    "                    price_list.append(int(tmp_dict['price']))\n",
    "                eta_list.append(int(tmp_dict['eta']))\n",
    "                mode_list.append(int(tmp_dict['transport_mode']))\n",
    "\n",
    "            # 将每个用户的推荐模型按顺序添加\n",
    "            mode_texts.append(' '.join(['word_{}'.format(mode) for mode in mode_list]))\n",
    "\n",
    "            # 将list转换成ndarray\n",
    "            distance_list = np.array(distance_list)\n",
    "            price_list = np.array(price_list)\n",
    "            eta_list = np.array(eta_list)\n",
    "            mode_list = np.array(mode_list, dtype='int')\n",
    "            \n",
    "            # 将有plans推荐的用户的mode置为1\n",
    "            mode_list_feas[i, mode_list] = 1\n",
    "\n",
    "            # 获取索引\n",
    "            distance_sort_idx = np.argsort(distance_list)\n",
    "            price_sort_idx = np.argsort(price_list)\n",
    "            eta_sort_idx = np.argsort(eta_list)\n",
    "\n",
    "            # 构建特征\n",
    "            max_distance[i] = distance_list[distance_sort_idx[-1]]\n",
    "            min_distance[i] = distance_list[distance_sort_idx[0]]\n",
    "            mean_distance[i] = np.mean(distance_list)\n",
    "            std_distance[i] = np.std(distance_list)\n",
    "\n",
    "            max_price[i] = price_list[price_sort_idx[-1]]\n",
    "            min_price[i] = price_list[price_sort_idx[0]]\n",
    "            mean_price[i] = np.mean(price_list)\n",
    "            std_price[i] = np.std(price_list)\n",
    "\n",
    "            max_eta[i] = eta_list[eta_sort_idx[-1]]\n",
    "            min_eta[i] = eta_list[eta_sort_idx[0]]\n",
    "            mean_eta[i] = np.mean(eta_list)\n",
    "            std_eta[i] = np.std(eta_list)\n",
    "\n",
    "            first_mode[i] = mode_list[0]\n",
    "\n",
    "            max_dis_mode[i] = mode_list[distance_sort_idx[-1]]\n",
    "            min_dis_mode[i] = mode_list[distance_sort_idx[0]]\n",
    "\n",
    "            max_price_mode[i] = mode_list[price_sort_idx[-1]]\n",
    "            min_price_mode[i] = mode_list[price_sort_idx[0]]\n",
    "\n",
    "            max_eta_mode[i] = mode_list[eta_sort_idx[-1]]\n",
    "            min_eta_mode[i] = mode_list[eta_sort_idx[0]]\n",
    "\n",
    "    # 将特征存储进DataFrame中\n",
    "    plan_feature_data = pd.DataFrame(mode_list_feas)\n",
    "    plan_feature_data.columns = ['mode_feas_{}'.format(i) for i in range(12)]\n",
    "\n",
    "    plan_feature_data['max_distance'] = max_distance\n",
    "    plan_feature_data['min_distance'] = min_distance\n",
    "    plan_feature_data['mean_distance'] = mean_distance\n",
    "    plan_feature_data['std_distance'] = std_distance\n",
    "\n",
    "    plan_feature_data['max_price'] = max_price\n",
    "    plan_feature_data['min_price'] = min_price\n",
    "    plan_feature_data['mean_price'] = mean_price\n",
    "    plan_feature_data['std_price'] = std_price\n",
    "\n",
    "    plan_feature_data['max_eta'] = max_eta\n",
    "    plan_feature_data['min_eta'] = min_eta\n",
    "    plan_feature_data['mean_eta'] = mean_eta\n",
    "    plan_feature_data['std_eta'] = std_eta\n",
    "\n",
    "    plan_feature_data['max_dis_mode'] = max_dis_mode\n",
    "    plan_feature_data['min_dis_mode'] = min_dis_mode\n",
    "    plan_feature_data['max_price_mode'] = max_price_mode\n",
    "    plan_feature_data['min_price_mode'] = min_price_mode\n",
    "    plan_feature_data['max_eta_mode'] = max_eta_mode\n",
    "    plan_feature_data['min_eta_mode'] = min_eta_mode\n",
    "\n",
    "    plan_feature_data['first_mode'] = first_mode\n",
    "\n",
    "    # tiidf提取特征\n",
    "    tfidf = TfidfVectorizer(ngram_range=(1, 2))\n",
    "    tfidf_vec = tfidf.fit_transform(mode_texts)\n",
    "    svd = TruncatedSVD(n_components=10, n_iter=20, random_state=2019)\n",
    "    mode_svd = svd.fit_transform(tfidf_vec)\n",
    "    \n",
    "    # 转换成dataframe\n",
    "    mode_svd = pd.DataFrame(mode_svd)\n",
    "    mode_svd.columns = ['svd_mode_{}'.format(i) for i in range(10)]\n",
    "\n",
    "    all_data = pd.concat([all_data, mode_svd, plan_feature_data], axis=1)\n",
    "    all_data = all_data.drop(['plans'], axis=1)\n",
    "    \n",
    "    return  all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4、抽取profiles数据集特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_profiles_feature(all_data):\n",
    "    profiles = pd.read_csv(\"data_set_phase1//profiles.csv\")\n",
    "\n",
    "    # 用于填充没有pid的用户\n",
    "    profiles_na = np.zeros(67)\n",
    "    profiles_na[0] = -1\n",
    "    profiles_na = pd.DataFrame(profiles_na.reshape(1, -1))\n",
    "    profiles_na.columns = profiles.columns\n",
    "    profiles = profiles.append(profiles_na)\n",
    "    \n",
    "    # 对特征进行奇异值分解，实现降维\n",
    "#     pi = profiles.drop(['pid'], axis=1).values\n",
    "#     svd = TruncatedSVD(n_components=60, n_iter=20, random_state=2019)\n",
    "#     profiles_svd = svd.fit_transform(pi)\n",
    "    \n",
    "    # 转换成dataframe\n",
    "#     profiles_svd = pd.DataFrame(profiles_svd)\n",
    "#     profiles_svd.columns = ['svd_profiles_{}'.format(i) for i in range(60)]\n",
    "#     profiles_svd['pid'] = profiles['pid'].values\n",
    "\n",
    "    # 合并数据集\n",
    "    all_data['pid'] = all_data['pid'].fillna(-1)\n",
    "    all_data = all_data.merge(profiles, on='pid', how='left')\n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5、抽取时间特征（req_time）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 距离国庆节的天数、查询的日期、周几、小时、分钟、是否是假期、是否是周末"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_time_feature(all_data):\n",
    "    \n",
    "    # 国庆嫁接\n",
    "    NatioinalDay = []\n",
    "    d1 = datetime.datetime(2018,10,1)\n",
    "    for i in range(all_data.shape[0]):\n",
    "        d2 = datetime.datetime.strptime(all_data.req_time[i].split(' ')[0], \"%Y-%m-%d\")\n",
    "        s = d2 - d1\n",
    "        NatioinalDay.append(s.days)\n",
    "    all_data['NatioinalDay'] = NatioinalDay\n",
    "    \n",
    "    all_data['req_time'] = pd.to_datetime(all_data['req_time'])\n",
    "    \n",
    "    # 查询日期\n",
    "    all_data['req_date'] = all_data['req_time'].dt.strftime(\"%m-%d\")\n",
    "    \n",
    "    # 是否是假期\n",
    "    all_data['isholiday'] = all_data['req_date'].isin(['10-01','10-02','10-03','10-04','10-05','10-06','10-07',]).astype(int)\n",
    "    \n",
    "    # 周几\n",
    "    all_data['dayofweek'] = all_data['req_time'].dt.dayofweek\n",
    "    \n",
    "    # 小时\n",
    "    all_data['req_hour'] = all_data['req_time'].dt.hour\n",
    "    \n",
    "    # 分钟\n",
    "    all_data['req_minute'] = all_data['req_time'].dt.minute\n",
    "    \n",
    "    # 是否是周末\n",
    "    all_data['isweekend']=0\n",
    "    all_data.isweekend[all_data['dayofweek']>4]=1\n",
    "    \n",
    "    all_data = all_data.drop(['req_date'], axis=1)\n",
    "    \n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6、提取pid特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 根据lgbm的特征重要度发现，pid是个强特，所以对pid进行特征提取\n",
    "### 1、统计每个pid出现的次数，将次数作为特征\n",
    "### 2、统计每个pid在每个类别中出现的次数（这个有问题）\n",
    "### 3、pid与时间特征的组合出现的次数\n",
    "### 4、pid与时间特征的组合在每个类别中出现的次数（这个有问题）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_pid_feature(all_data):\n",
    "    \n",
    "    # 统计每个pid出现的次数\n",
    "    pid_counts = pd.DataFrame()\n",
    "    counts = all_data['pid'].value_counts()\n",
    "    index = counts.index\n",
    "    pid_counts['pid'] = index\n",
    "    pid_counts['pid_counts'] = list(counts)\n",
    "    \n",
    "    # pid与first_mode组合出现次数\n",
    "    pidCombineFirstM0 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==0].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM0['pid'] = index\n",
    "    pidCombineFirstM0['pid_F0_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM1 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==1].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM1['pid'] = index\n",
    "    pidCombineFirstM1['pid_F1_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM2 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==2].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM2['pid'] = index\n",
    "    pidCombineFirstM2['pid_F2_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM3 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==3].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM3['pid'] = index\n",
    "    pidCombineFirstM3['pid_F3_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM4 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==4].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM4['pid'] = index\n",
    "    pidCombineFirstM4['pid_F4_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM5 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==5].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM5['pid'] = index\n",
    "    pidCombineFirstM5['pid_F5_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM6 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==6].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM6['pid'] = index\n",
    "    pidCombineFirstM6['pid_F6_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM7 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==7].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM7['pid'] = index\n",
    "    pidCombineFirstM7['pid_F7_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM8 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==8].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM8['pid'] = index\n",
    "    pidCombineFirstM8['pid_F8_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM9 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==9].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM9['pid'] = index\n",
    "    pidCombineFirstM9['pid_F9_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM10 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==10].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM10['pid'] = index\n",
    "    pidCombineFirstM10['pid_F10_counts'] = list(counts)\n",
    "    \n",
    "    pidCombineFirstM11 = pd.DataFrame()\n",
    "    counts = all_data.pid[all_data['first_mode']==11].value_counts()\n",
    "    index = counts.index\n",
    "    pidCombineFirstM11['pid'] = index\n",
    "    pidCombineFirstM11['pid_F11_counts'] = list(counts)\n",
    "    \n",
    "    # pid与时间特征的组合出现次数，按是否是节假日进行组合\n",
    "    pidCombineHoliday0 = pd.DataFrame()\n",
    "    counts_0 = all_data.pid[all_data['isholiday']==0].value_counts()\n",
    "    index_0 = counts_0.index\n",
    "    pidCombineHoliday0['pid'] = index_0\n",
    "    pidCombineHoliday0['pid_H0_counts'] = list(counts_0)\n",
    "    \n",
    "    pidCombineHoliday1 = pd.DataFrame()\n",
    "    counts_1 = all_data.pid[all_data['isholiday']==1].value_counts()\n",
    "    index_1 = counts_1.index\n",
    "    pidCombineHoliday1['pid'] = index_1\n",
    "    pidCombineHoliday1['pid_H1_counts'] = list(counts_1)\n",
    "    \n",
    "    # pid与时间特征的组合出现次数，按是否是周末进行组合\n",
    "    pidCombineWeekend0 = pd.DataFrame()\n",
    "    counts_0 = all_data.pid[all_data['isweekend']==0].value_counts()\n",
    "    index_0 = counts_0.index\n",
    "    pidCombineWeekend0['pid'] = index_0\n",
    "    pidCombineWeekend0['pid_W0_counts'] = list(counts_0)\n",
    "    \n",
    "    pidCombineWeekend1 = pd.DataFrame()\n",
    "    counts_1 = all_data.pid[all_data['isweekend']==1].value_counts()\n",
    "    index_1 = counts_1.index\n",
    "    pidCombineWeekend1['pid'] = index_1\n",
    "    pidCombineWeekend1['pid_W1_counts'] = list(counts_1)\n",
    "    \n",
    "    #合并dataframe\n",
    "    all_data = all_data.merge(pidCombineFirstM0, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM1, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM2, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM3, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM4, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM5, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM6, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM7, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM8, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM9, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM10, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineFirstM11, on='pid', how='left')\n",
    "    all_data = all_data.merge(pid_counts, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineHoliday0, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineHoliday1, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineWeekend0, on='pid', how='left')\n",
    "    all_data = all_data.merge(pidCombineWeekend1, on='pid', how='left')\n",
    "    \n",
    "    # 填充缺失值\n",
    "    all_data['pid_F0_counts'] = all_data['pid_F0_counts'].fillna(0)\n",
    "    all_data['pid_F1_counts'] = all_data['pid_F1_counts'].fillna(0)\n",
    "    all_data['pid_F2_counts'] = all_data['pid_F2_counts'].fillna(0)\n",
    "    all_data['pid_F3_counts'] = all_data['pid_F3_counts'].fillna(0)\n",
    "    all_data['pid_F4_counts'] = all_data['pid_F4_counts'].fillna(0)\n",
    "    all_data['pid_F5_counts'] = all_data['pid_F5_counts'].fillna(0)\n",
    "    all_data['pid_F6_counts'] = all_data['pid_F6_counts'].fillna(0)\n",
    "    all_data['pid_F7_counts'] = all_data['pid_F7_counts'].fillna(0)\n",
    "    all_data['pid_F8_counts'] = all_data['pid_F8_counts'].fillna(0)\n",
    "    all_data['pid_F9_counts'] = all_data['pid_F9_counts'].fillna(0)\n",
    "    all_data['pid_F10_counts'] = all_data['pid_F10_counts'].fillna(0)\n",
    "    all_data['pid_F11_counts'] = all_data['pid_F11_counts'].fillna(0)\n",
    "    all_data['pid_H0_counts'] = all_data['pid_H0_counts'].fillna(0)\n",
    "    all_data['pid_H1_counts'] = all_data['pid_H1_counts'].fillna(0)\n",
    "    all_data['pid_W0_counts'] = all_data['pid_W0_counts'].fillna(0)\n",
    "    all_data['pid_W1_counts'] = all_data['pid_W1_counts'].fillna(0)\n",
    "    \n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7、切分数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(all_data):\n",
    "    train_data = all_data[all_data['click_mode']!=-1]\n",
    "    test_data = all_data[all_data['click_mode']==-1]\n",
    "    test_data = test_data.drop(['click_mode'], axis=1)\n",
    "    submit = test_data[['sid']].copy()\n",
    "    \n",
    "    train_data = train_data.drop(['sid', 'req_time'], axis=1)\n",
    "    train_y = train_data['click_mode']\n",
    "    train_x = train_data.drop(['click_mode'], axis=1)\n",
    "    test_x = test_data.drop(['sid', 'req_time'], axis=1)\n",
    "    \n",
    "    return train_x, train_y, test_x, submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "594358it [01:31, 6484.03it/s]\n",
      "E:\\SoftWare\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "all_data = merge_data()\n",
    "all_data = gen_od_feature(all_data)\n",
    "all_data = gen_plan_feature(all_data)\n",
    "all_data = gen_profiles_feature(all_data)\n",
    "all_data = gen_time_feature(all_data)\n",
    "all_data = gen_pid_feature(all_data)\n",
    "train_x, train_y, test_x, submit = train_test_split(all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8、模型训练&验证&提交"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\SoftWare\\Anaconda\\lib\\site-packages\\lightgbm\\basic.py:1205: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "E:\\SoftWare\\Anaconda\\lib\\site-packages\\lightgbm\\basic.py:762: UserWarning: categorical_feature in param dict is overridden.\n",
      "  warnings.warn('categorical_feature in param dict is overridden.')\n",
      "E:\\SoftWare\\Anaconda\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[50]\tvalid_0's multi_logloss: 1.01235\tvalid_0's weighted-f1-score: 0.671802\n",
      "[100]\tvalid_0's multi_logloss: 0.917836\tvalid_0's weighted-f1-score: 0.677159\n",
      "[150]\tvalid_0's multi_logloss: 0.897418\tvalid_0's weighted-f1-score: 0.679313\n",
      "[200]\tvalid_0's multi_logloss: 0.892203\tvalid_0's weighted-f1-score: 0.679891\n",
      "[250]\tvalid_0's multi_logloss: 0.891986\tvalid_0's weighted-f1-score: 0.679858\n",
      "Early stopping, best iteration is:\n",
      "[204]\tvalid_0's multi_logloss: 0.892125\tvalid_0's weighted-f1-score: 0.680097\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[50]\tvalid_0's multi_logloss: 1.00907\tvalid_0's weighted-f1-score: 0.673015\n",
      "[100]\tvalid_0's multi_logloss: 0.91363\tvalid_0's weighted-f1-score: 0.678086\n",
      "[150]\tvalid_0's multi_logloss: 0.892745\tvalid_0's weighted-f1-score: 0.68014\n",
      "[200]\tvalid_0's multi_logloss: 0.887679\tvalid_0's weighted-f1-score: 0.680696\n",
      "[250]\tvalid_0's multi_logloss: 0.887403\tvalid_0's weighted-f1-score: 0.680739\n",
      "Early stopping, best iteration is:\n",
      "[232]\tvalid_0's multi_logloss: 0.887376\tvalid_0's weighted-f1-score: 0.681002\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[50]\tvalid_0's multi_logloss: 1.01059\tvalid_0's weighted-f1-score: 0.671022\n",
      "[100]\tvalid_0's multi_logloss: 0.915297\tvalid_0's weighted-f1-score: 0.676169\n",
      "[150]\tvalid_0's multi_logloss: 0.894666\tvalid_0's weighted-f1-score: 0.678\n",
      "[200]\tvalid_0's multi_logloss: 0.889319\tvalid_0's weighted-f1-score: 0.678579\n",
      "Early stopping, best iteration is:\n",
      "[167]\tvalid_0's multi_logloss: 0.892028\tvalid_0's weighted-f1-score: 0.678759\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[50]\tvalid_0's multi_logloss: 1.01441\tvalid_0's weighted-f1-score: 0.669752\n",
      "[100]\tvalid_0's multi_logloss: 0.919231\tvalid_0's weighted-f1-score: 0.676146\n",
      "[150]\tvalid_0's multi_logloss: 0.898659\tvalid_0's weighted-f1-score: 0.678265\n",
      "[200]\tvalid_0's multi_logloss: 0.893251\tvalid_0's weighted-f1-score: 0.679139\n",
      "[250]\tvalid_0's multi_logloss: 0.892538\tvalid_0's weighted-f1-score: 0.679463\n",
      "Early stopping, best iteration is:\n",
      "[241]\tvalid_0's multi_logloss: 0.892464\tvalid_0's weighted-f1-score: 0.679418\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[50]\tvalid_0's multi_logloss: 1.01131\tvalid_0's weighted-f1-score: 0.671112\n",
      "[100]\tvalid_0's multi_logloss: 0.916127\tvalid_0's weighted-f1-score: 0.676558\n",
      "[150]\tvalid_0's multi_logloss: 0.895547\tvalid_0's weighted-f1-score: 0.67822\n",
      "[200]\tvalid_0's multi_logloss: 0.890399\tvalid_0's weighted-f1-score: 0.678796\n",
      "[250]\tvalid_0's multi_logloss: 0.889788\tvalid_0's weighted-f1-score: 0.679359\n",
      "Early stopping, best iteration is:\n",
      "[238]\tvalid_0's multi_logloss: 0.889686\tvalid_0's weighted-f1-score: 0.67927\n",
      "cv f1_score: 0.6797091565671113\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from time import gmtime, strftime\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "def f1_weighted(y_pred, train_data):\n",
    "    y_true = train_data.label\n",
    "    y_pred = y_pred.reshape(12, -1).T\n",
    "    y_pred = np.argmax(y_pred, axis=1)\n",
    "    f1 = f1_score(y_true, y_pred, average='weighted')\n",
    "    return 'weighted-f1-score', f1, True\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)\n",
    "\n",
    "lgb_paras = {\n",
    "    'objective': 'multiclass',\n",
    "    'metrics': 'multiclass',\n",
    "    'learning_rate': 0.05,\n",
    "    'num_leaves': 31,\n",
    "    'lambda_l1': 0.01,\n",
    "    'lambda_l2': 10,\n",
    "    'num_class': 12,\n",
    "    'seed': 2019,\n",
    "    'feature_fraction': 0.8,\n",
    "    'bagging_fraction': 0.8,\n",
    "    'bagging_freq': 4\n",
    "}\n",
    "\n",
    "categorical_feature = ['pid', 'max_dis_mode', 'min_dis_mode', 'max_price_mode', 'min_price_mode',\n",
    "                       'max_eta_mode', 'min_eta_mode', 'first_mode']\n",
    "scores = []\n",
    "result_proba = []\n",
    "for tra_idx, val_idx in kfold.split(train_x, train_y):\n",
    "    tra_x, tra_y, val_x, val_y = train_x.iloc[tra_idx], train_y[tra_idx], train_x.iloc[val_idx], train_y[val_idx]\n",
    "    train_set = lgb.Dataset(tra_x, tra_y, categorical_feature=categorical_feature)\n",
    "    val_set = lgb.Dataset(val_x, val_y, categorical_feature=categorical_feature)\n",
    "    lgb_model = lgb.train(lgb_paras, train_set, valid_sets=[val_set], early_stopping_rounds=50, num_boost_round=40000, verbose_eval=50, feval=f1_weighted)\n",
    "    val_pred = np.argmax(lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration), axis=1)\n",
    "    val_score = f1_score(val_y, val_pred, average='weighted')\n",
    "    result_proba.append(lgb_model.predict(test_x, num_iteration=lgb_model.best_iteration))\n",
    "    scores.append(val_score)\n",
    "print('cv f1_score:', np.mean(scores))\n",
    "pred_test = np.argmax(np.mean(result_proba, axis=0), axis=1)\n",
    "\n",
    "# 提交结果\n",
    "now_time = strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "submit['recommend_mode'] = pred_test\n",
    "submit.to_csv('submission_{}.csv'.format(now_time), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征重要度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'lightgbm' has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-406a62dbba9d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mimp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mimp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'feature'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtra_x\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mimp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'imp'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mimp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'imp'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mascending\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mimp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: module 'lightgbm' has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "imp = pd.DataFrame()\n",
    "imp['feature'] = tra_x.columns\n",
    "imp['imp'] = lgb.feature_importances_\n",
    "imp = imp.sort_values('imp', ascending = False)\n",
    "imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6833997071807066"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre = lgb.predict(valid_x)\n",
    "f1_score(valid_y, pre, average='weighted')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 提交结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = train_x.drop(['req_time'], axis=1)\n",
    "y = train_y.drop(['req_time'], axis=1)\n",
    "\n",
    "lgb.n_estimators = lgb.best_iteration_\n",
    "lgb.fit(x, y, categorical_feature=categorical_feature)\n",
    "pred_test = lgb.predict(test_x)\n",
    "\n",
    "# 提交结果\n",
    "now_time = strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "submit['recommend_mode'] = pred_test\n",
    "submit.to_csv('submission_{}.csv'.format(now_time), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
